package com.miyu.client1.service.magic.impl;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import us.codecraft.webmagic.*;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.List;

public class tu {
    public static void main(String[] args) {
        JSONObject data = new JSONObject();
        data.put("type","1");
        //data.put("downType","m");
         data.put("downType","t");
        String url ="https://xn--qbt00o3ns2fk.xyz/";
        Request request = new Request();
        request.setUrl(url);
        request.setCharset("utf-8");
        request.putExtra("info",data);


        Spider.create(new TUspider()).addRequest(request)
                .addPipeline(new downLoadFile())
                .thread(50).run();
    }

    static class TUspider implements PageProcessor {

        @Override
        public void process(Page page) {
            String url=page.getUrl().get();
            JSONObject ext  = page.getRequest().getExtra("info");
            if(ext.getString("type").equals("2")){
                String name = page.getHtml().xpath("//div[@class='artical-content']/h3/a/text()").get();
                JSONArray ja = new JSONArray();
                    //获取所有图片链接
                    List<Selectable> nodes = page.getHtml().xpath("//div[@class='artical-content']").nodes();
                    for (Selectable li:nodes) {
                        JSONObject jo = new JSONObject();
                        String original = li.xpath("//img/@data-original").get();
                        jo.put("srcUrl",original);
                        jo.put("name",name);
                        ja.add(jo);
                    }
                    page.putField("name",name);
                    page.putField("imgSrc",ja);

            }else if(ext.getString("type").equals("1")){
                getLikes(page,ext);
                JSONObject data = new JSONObject();
                data.put("type","1");
                String s = page.getHtml().xpath("///ul[@class='stui-page text-center cleafix']/li[4]/a").links().all().get(0);
                if(s.indexOf("javascript")<0){
                    data.put("type","1");
                    data.put("downType",ext.getString("downType"));
                    Request request = new Request();
                    request.setUrl(s);

                    request.setCharset("utf-8");
                    request.putExtra("info",data);
                    page.addTargetRequest(request);
                    page.setSkip(true);
                }
            }
        }

        private void getLikes(Page page,JSONObject ext) {
            //当前数据
            List<String> likes = page.getHtml().xpath("//div[@id='main']/ul[@id='tiles']/li").xpath("//li/@onclick").all();
            JSONObject data = new JSONObject();
            data.put("type","2");
            data.put("downType",ext.getString("downType"));
            for (String plike: likes) {
                Request request = new Request();
                request.setUrl("https://xn--qbt00o3ns2fk.xyz/"+plike.substring(plike.indexOf("'")+1,plike.lastIndexOf("'")));
                request.putExtra("info",data);
                page.addTargetRequest(request);
                page.setSkip(true);
            }
        }

        @Override
        public Site getSite() {
            return this.site;
        }
        //爬虫配置信息设置
        private Site site = new Site()
                .setCharset("utf-8")    //设置编码
                .addCookie("_ym_uid","1622118203527611659")
                .addCookie("_ym","1622118203")
                .addCookie(" _ym_isad","1")
                .setSleepTime(3)        //设置抓取间隔
                .setTimeOut(10000)         //设置超时时间
                .setRetrySleepTime(300)      //设置重试时间
                .setRetryTimes(3)
                .setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36 Edg/87.0.664.52");       //设置重试次数

    }
    static class  downLoadFile implements Pipeline {

        @Override
        public void process(ResultItems resultItems, Task task) {
            String fileName = resultItems.get("name").toString().trim();
            //路径
            String filePath = "H:\\爬虫数据\\图片\\tu\\";
            JSONArray ja = resultItems.get("imgSrc");
            //先创建文件夹
            File fp = new File(filePath+fileName);
            if (!fp.exists()) {
                fp.mkdirs();// 能创建多级目录
            };
            for(int i=0;i<ja.size();i++){
                JSONObject j = (JSONObject) ja.get(i);
                String link = j.getString("srcUrl");
                URL url = null;
                try {
                    url = new URL(link);
                    URLConnection con = url.openConnection();

                    InputStream inStream = con.getInputStream();
                    if(inStream.available()==0)
                        continue;
                    ByteArrayOutputStream outStream = new ByteArrayOutputStream();
                    byte[] buf = new byte[1024];
                    int len = 0;
                    while((len = inStream.read(buf)) != -1){
                        outStream.write(buf,0,len);
                    }
                    String kz = link.substring(link.lastIndexOf("."),link.length());
                    String FN = filePath+fileName+"//"+i+kz;
                    File file = new File(FN.trim());
                    if(!file.exists()) {
                        file.createNewFile();
                    }
                    FileOutputStream op = new FileOutputStream(file);
                    op.write(outStream.toByteArray());
                    inStream.close();
                    outStream.close();
                    op.close();
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }


    }
}
